// --------------------------------------------------------------------------------------------------------------------
// <copyright file="MetricsController.cs" company="Microsoft Corporation">
// The MIT License (MIT)
// 
// Copyright (c) 2014, Microsoft Corporation
// 
// Permission is hereby granted, free of charge, to any person obtaining a copy
//  of this software and associated documentation files (the "Software"), to deal
//  in the Software without restriction, including without limitation the rights
//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
//  copies of the Software, and to permit persons to whom the Software is
//  furnished to do so, subject to the following conditions:
// 
// The above copyright notice and this permission notice shall be included in
//  all copies or substantial portions of the Software.
// 
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
//  THE SOFTWARE.
// </copyright>
// --------------------------------------------------------------------------------------------------------------------
namespace Test.Robotics.Metrics
{
    using System;
    using System.Collections.Generic;
    using System.Linq;
    using System.Runtime.Serialization;
    using Microsoft.Robotics.Runtime;
    using MicrosoftResearch.Infer.Distributions;

    /// <summary>
    /// Agent that controls the execution of multiple metric runs, whether online or offline.
    /// It consumes MetricsResultMessages which are the output of a metrics calculator agent and keeps a running tally of errors and weights for
    /// calculating weighted averages.  It displays results to the screen and it re-publishes the original metrics messages for consumption by other
    /// agents (such as for logging to a database).
    /// It will perform multiple iterations by deactivating and reactivating the producer agent.
    /// </summary>
    [DataContract]
    public class MetricsController : ConsumerProducerAgent<MetricsResultMessage, MetricsResultMessage>, ISubscriptionReceiver<CompletionMessage>, IMessagePublisher<CompletionMessage>
    {
        /// <summary>
        /// Default number of runs to perform in a single session
        /// </summary>
        private const int DefaultNumberOfRuns = 100;

        /// <summary>
        /// The set of agents that collectively indicate when a single run has finished. 
        /// </summary>
        [DataMember(Name = "CompletionVoters")]
        private readonly string[] completionVoters;

        /// <summary>
        /// The number of runs that will complete a test pass
        /// </summary>
        [DataMember(Name = "Iterations")]
        private readonly int iterationCount;

        /// <summary>
        /// The current iteration count.
        /// </summary>
        private int iteration;

        /// <summary>
        /// The sum of weighted errors
        /// </summary>
        private Dictionary<string, double> sumOfErrors;

        /// <summary>
        /// The normalizer for the error sum.
        /// </summary>
        private Dictionary<string, double> errorNormalizer;

        /// <summary>
        /// The number of pose errors we have received. Used to compute the Freiburg error.
        /// </summary>
        private Dictionary<string, int> countOfErrors;

        /// <summary>
        /// The estimator for the accuracy mean and deviation, for each type of measurement.
        /// </summary>
        private Dictionary<string, GaussianEstimator> accuracyEstimators;

        /// <summary>
        /// The count of completion agents that have indicated the end of the run so far.
        /// </summary>
        private int completionVotes;

        /// <summary>
        /// This is how we communicate that we completed the session.
        /// </summary>
        private IForwarder<CompletionMessage> completionPort;

        /// <summary>
        /// Initializes a new instance of the MetricsController class.
        /// </summary>
        /// <param name="name">The name of the agent.</param>
        /// <param name="calculatorAgent">The metrics calculator agent that generates metrics results</param>
        /// <param name="completionAgents">The set of agents used to decide when a run has completed</param>
        /// <param name="iterations">The number of iterations to run.</param>
        public MetricsController(string name, IProducer<MetricsResultMessage> calculatorAgent, IEnumerable<IProducer<CompletionMessage>> completionAgents, int iterations)
            : base(name, calculatorAgent.Name)
        {
            this.completionVoters = completionAgents.Select(p => p.Name).ToArray();
            this.iterationCount = iterations;
        }

        /// <summary>
        /// De-facto constructor for the class, since the actual constructor is not called when the instance is created through deserialization.
        /// </summary>
        /// <param name="locator">The locator to use to find other agents.</param>
        public override void Initialize(AgentLocator locator)
        {
            base.Initialize(locator);            
            foreach (string completionAgent in this.completionVoters)
            {
                IAgentHost agent = locator.GetAgent(completionAgent);
                agent.Subscribe<CompletionMessage>(this);
            }

            this.sumOfErrors = new Dictionary<string, double>();
            this.errorNormalizer = new Dictionary<string, double>();
            this.countOfErrors = new Dictionary<string, int>();
            this.accuracyEstimators = new Dictionary<string, GaussianEstimator>();
        }

        /// <summary>
        /// Called to give us the port to use for publishing completion messages.
        /// </summary>
        /// <param name="forwarder">The completion port.</param>
        public void InitializePublisher(IForwarder<CompletionMessage> forwarder)
        {
            this.completionPort = forwarder;
        }

        /// <summary>
        /// Called whenever a metrics estimate is available. 
        /// Output results to the console and republish the message.
        /// </summary>
        /// <param name="message">Result message</param>
        [ReceiverConfiguration(BlockIfFull = true)]
        public override void Receive(MetricsResultMessage message)
        {
            double error = message.Accuracy * message.Weight;
            
            if (!this.sumOfErrors.ContainsKey(message.Type))
            {
                this.sumOfErrors[message.Type] = error;
                this.errorNormalizer[message.Type] = message.Weight;
                this.countOfErrors[message.Type] = 1;
            }
            else
            {
                this.sumOfErrors[message.Type] += error;
                this.errorNormalizer[message.Type] += message.Weight;
                this.countOfErrors[message.Type]++;
            }

            Console.WriteLine(
                "P{0}: {1} = {2:F4} [Freiburg+ = {3:F4}] ({4:F4}/{5:F4} sigma = {6:F4})", 
                message.GroundTruthId, 
                message.Type, 
                message.Accuracy,
                this.sumOfErrors[message.Type] / this.countOfErrors[message.Type],
                error,
                message.Weight, 
                message.Precision);

            this.Publisher.Post(message);

            if (message.IsFinalMessage)
            {
                this.SummarizeResults(false, string.Empty);
            }
        }

        /// <summary>
        /// Called whenever one of the completion voters finished 
        /// </summary>
        /// <param name="message">A completion message indicating that a file player finished playing.</param>
        public void Receive(CompletionMessage message)
        {
            this.completionVotes++;
            if (this.completionVotes == this.completionVoters.Length)
            {
                this.SummarizeResults(message.IsFailure, message.FailureReason);
            }
        }

        /// <summary>
        /// Summarize this run and start a new one if more iterations are indicated
        /// </summary>
        /// <param name="isFailure">Indicates if this indicates a test failure.</param>
        /// <param name="failureReason">Human-friendly description of failure.</param>
        private void SummarizeResults(bool isFailure, string failureReason)
        {
            // all files completed, should we loop again?
            this.iteration++;

            // write the statistics for the run
            Console.WriteLine("Run {0} completed.", this.iteration);
            foreach (KeyValuePair<string, double> totalError in this.sumOfErrors)
            {
                Console.WriteLine("\t{0} accuracy = {1:F4}", totalError.Key, totalError.Value / this.errorNormalizer[totalError.Key]);
                if (!this.accuracyEstimators.ContainsKey(totalError.Key))
                {
                    this.accuracyEstimators[totalError.Key] = new GaussianEstimator();
                }

                this.accuracyEstimators[totalError.Key].Add(totalError.Value / this.errorNormalizer[totalError.Key]);
            }

            Console.WriteLine();

            this.sumOfErrors.Clear();
            this.errorNormalizer.Clear();

            // if this was the last run, write the aggregate statistics
            if (this.iterationCount == this.iteration)
            {
                Console.WriteLine("Metrics run completed with {0} iterations. Results:", this.iteration);
                foreach (KeyValuePair<string, GaussianEstimator> estimator in this.accuracyEstimators)
                {
                    double accuracy;
                    double variance;
                    Gaussian g = estimator.Value.GetDistribution(new Gaussian());
                    g.GetMeanAndVariance(out accuracy, out variance);
                    Console.WriteLine("\t{0} accuracy = {1:F4}, precision = {2:F4}", estimator.Key, accuracy, Math.Sqrt(variance));
                }

                // let the host know we are done
                this.completionPort.Post(new CompletionMessage(this.Name, isFailure, failureReason));
            }
            else
            {
                // restart the run
                this.completionVotes = 0;
                this.ProducerAgent.Deactivate();
                this.ProducerAgent.Activate();
            }
        }
    }
}
